import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}

object context {
  def main(args: Array[String]): Unit = {
    //词频统计
    //创建Spark Context对象
    //创建配置对象
    val conf: SparkConf = new SparkConf().setAppName("wordscount").setMaster("local[2]")
    val sc = new SparkContext(conf)
    //设置日志的级别
    sc.setLogLevel("WARN")
    val rdd1: RDD[Int] = sc.parallelize(1 to 10) //8
    val rdd2: RDD[Int] = sc.parallelize(1 to 10, 3) //3

    val rdd3: RDD[Int] = sc.makeRDD(1 to 10) //底层是parallelize //8
    val rdd4: RDD[Int] = sc.makeRDD(1 to 10, 4) //4

    //RDD[一行行的数据]  文件很少，分区至少2
//    val rdd5: RDD[String] = sc.textFile("data/input/words.txt") //2
//    val rdd6: RDD[String] = sc.textFile("data/input/words.txt", 3) //3
    //RDD[一行行的数据]  文件夹，有多少个文件，就有多少个分区，指定分区值无效！
    val rdd7: RDD[String] = sc.textFile("data/input/ratings10") //10
    val rdd8: RDD[String] = sc.textFile("data/input/ratings10", 3) //10
    //RDD[(文件名, 一行行的数据),(文件名, 一行行的数据)....]
    val rdd9: RDD[(String, String)] = sc.wholeTextFiles("data/input/ratings10") //2
    val rdd10: RDD[(String, String)] = sc.wholeTextFiles("data/input/ratings10", 3) //3

    println(rdd1.getNumPartitions) //8 //底层partitions.length
    println(rdd2.partitions.length) //3
    println(rdd3.getNumPartitions) //8
    println(rdd4.getNumPartitions) //4
//    println(rdd5.getNumPartitions) //2
//    println(rdd6.getNumPartitions) //3
    println(rdd7.getNumPartitions) //10
    println(rdd8.getNumPartitions) //10
    println(rdd9.getNumPartitions) //2
    println(rdd10.getNumPartitions) //3
  }
}

